const puppeteer = require('puppeteer');
const ora = require('ora');
const FakeUseragent = require('fake-useragent');
const fs = require('fs');
const Paint = require('../tool/console.color');


let targetWebsiteURL = 'https://zz.lianjia.com/ershoufang/';
let pageOption = {
  timeout: 0,
  waitUntil: 'domcontentloaded'
}

// 文件是否存在
function fileIsExist(filePath) {
  try {
    fs.readFileSync(filePath);
    return true;
  } catch {
    return false;
  }
}

// 获取街道数据
async function getStreetData(fileName) {
  // 创建浏览器窗口
  Paint('新建浏览器中...', -1);
  const browser = await puppeteer.launch({
    headless: true
  })

  // 创建标签页
  Paint('新建页面中...', -1);
  const page = await browser.newPage();

  // 进入待爬取页面
  await page.goto(targetWebsiteURL);

  let arealist = await page.$$eval('div[data-role=ershoufang]>div:nth-child(1)>a', els => els.map(item => ({
    name: item.innerHTML,
    url: item.href
  })));

  let allStreetNameList = [];
  let processLoading = ora('0%').start();

  for (let i in arealist) {
    let other_page = await browser.newPage();
    other_page.setUserAgent(FakeUseragent());
    await other_page.goto(arealist[i].url, pageOption)
    let streetlist = await other_page.$$eval('div[data-role=ershoufang]>div:nth-child(2)>a', els => els.map(item => ({
      name: item.innerHTML,
      url: item.href
    })))
    let listAfterFilter = streetlist.filter(item => !allStreetNameList.find(val => item.name === val.name));
    arealist[i].street = {}
    listAfterFilter.forEach(item => {
      arealist[i].street[item.name] = item.url;
    })
    allStreetNameList = [...allStreetNameList, ...listAfterFilter];
    // * 1. 用自定义的颜色打印函数
    // Paint(`${+i + 1}/${arealist.length} ${arealist[i].name} 记录完毕！`, -1);

    // * 2. 使用ora显示进度
    processLoading.text = parseInt((+i + 1) / arealist.length * 100) + '%';
  }

  fs.writeFile(fileName, JSON.stringify(allStreetNameList), err => {
    if (err) {
      console.error(err);
    } else {
      // * 1. 用自定义的颜色打印函数
      // Paint('街道数据收集成功！');

      // * 2. 使用ora显示进度
      processLoading.succeed('街道数据收集成功！');
    }
  })

  await browser.close();
}

// 获取每个街道房源信息
async function getRoomInfoByPreStreet(fileName) {
  let data = fs.readFileSync(fileName, {
    encoding: 'utf-8'
  })
  let streetlist = JSON.parse(data);
  Paint('新建浏览器中...', -1);
  const browser = await puppeteer.launch();
  let successNum = 0;
  let failureNum = 0;
  for (let streetIndex = 0; streetIndex < streetlist.length; streetIndex++) {
    if (fileIsExist(__dirname + `/data/${streetlist[streetIndex].name}.json`)) continue;
    Paint('新建页面中...', -1);
    let [page] = await browser.pages();
    await page.setUserAgent(FakeUseragent());
    // await page.evaluateOnNewDocument(() => {
    //   window.navigator.chrome = {
    //     runtime: {}
    //   };
    // });
    await page.goto(streetlist[streetIndex].url, pageOption);

    let maxNum = await page.$$eval('div.resultDes.clear > h2 > span', elements => {
      if (elements.length) {
        return parseInt(elements[0].innerText.trim());
      } else {
        return 0;
      }
    });

    let maxPage = await page.$$eval('div[comp-module=page]', els => {
      if (els.length) {
        return JSON.parse(els[0].getAttribute('page-data')).totalPage;
      } else {
        return 0;
      }
    })
    let info = [];
    let hasHouse = false;
    for (let i = 1; i <= maxPage; i++) {
      if (i > 1) {
        await page.goto(`${streetlist[streetIndex].url}pg${i}`, pageOption);
      }
      let pageSize = 30;
      if (i === maxPage) {
        pageSize = maxNum % 30;
      }
      for (let index = 1; index <= pageSize; index++) {
        try {
          let address = await page.$$eval(`#content > div.leftContent > ul > li:nth-child(${index}) > div.info.clear > div.flood > div > a`, elements => elements.map(val => val.innerHTML.trim()));
          address = address.join(' - ');

          let houseInfo = await page.$$eval(`#content > div.leftContent > ul > li:nth-child(${index}) > div.info.clear > div.address > div`, elements => elements[0].innerText);

          let price = await page.$$eval(`#content > div.leftContent > ul > li:nth-child(${index}) > div.info.clear > div.priceInfo span`, elements => elements.map(val => val.innerText));
          price = price.join('万 - ');

          info.push({
            address,
            houseInfo,
            price
          });
        } catch {
          hasHouse = info.length;
          info = [];
          i = 9999;
          break;
        }
      }
      if (i <= maxPage) {
        Paint(`${streetlist[streetIndex].name}\t${i}/${maxPage}页 抓取完成！`, -1);
      }
    }
    if (info.length) {
      successNum++;
      fs.writeFile(__dirname + `/data/${streetlist[streetIndex].name}.json`, JSON.stringify(info), err => {
        if (err) {
          console.error(err);
        } else {
          Paint(`${streetlist[streetIndex].name} ${maxNum} 套房源信息保存完毕！${streetIndex}/${streetlist.length}`);
        }
      })
    } else if (hasHouse) {
      failureNum++;
      Paint(`${streetlist[streetIndex].name} 房源信息爬取失败！`, 2);
    } else {
      Paint(`${streetlist[streetIndex].name} 房源信息为空！`, 1);
    }
  }
  Paint(`本次房源信息收集完毕！`, -1);
  Paint(`收集成功 ${successNum} 个街道`);
  Paint(`收集失败 ${failureNum} 个街道`, 2);
  await browser.close();
}

let fileName = 'streetlist.json'

async function test() {
  Paint('新建浏览器中...');
  let browser = await puppeteer.launch();

  Paint('新建页面中...');
  let page = await browser.newPage();
  await page.setUserAgent(FakeUseragent());

  await page.goto('https://zz.lianjia.com/ershoufang/bishagang/', pageOption);
  let process = 0;
  let spinner = ora(`${process}%`).start();
  spinner.color = 'blue';
  for (let index = 1; index <= 30; index++) {
    let address = await page.$$eval(`ul[log-mod=list] > li:nth-child(${index}) > div.info.clear > div.flood > div > a`, elements => elements.map(val => val.innerHTML.trim()));
    address = address.join(' - ');

    let houseInfo = await page.$$eval(`ul[log-mod=list] > li:nth-child(${index}) > div.info.clear > div.address > div`, elements => elements[0].innerText);

    let price = await page.$$eval(`ul[log-mod=list] > li:nth-child(${index}) > div.info.clear > div.priceInfo span`, elements => elements.map(val => val.innerText));
    price = price.join('万 - ');

    spinner.text = `${parseInt(index/30*100)}%`;
  }
  spinner.succeed('bingo!');

  await browser.close();
}

// test();

// getStreetData('oraText.json');

// getRoomInfoByPreStreet(fileName);
// console.log(fileIsExist(__dirname + '/data/houseInf1o.json'))

// (async function () {
//   Paint('新建浏览器中...');
//   let ip = '182.84.145.58'
//   let browser = await puppeteer.launch({
//     headless: false,
//     args: [`-proxy-server=${ip}`]
//   });

//   Paint('新建页面中...');
//   let [page] = await browser.pages();
//   await page.setUserAgent(FakeUseragent());

//   await page.goto('https://www.baidu.com', pageOption);
// })
